{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import sys\n",
    "import os\n",
    "from nbfinder import NotebookFinder\n",
    "sys.meta_path.append(NotebookFinder())\n",
    "\n",
    "import netCDF4 as nc\n",
    "from netCDF4 import MFDataset\n",
    "from os import listdir, system\n",
    "from os.path import isfile, join, isdir\n",
    "import numpy as np\n",
    "import imp\n",
    "import itertools\n",
    "import numpy as np\n",
    "import os\n",
    "import sys\n",
    "import time\n",
    "import inspect\n",
    "import copy\n",
    "from util import get_camfiles, normalize,convert_nc_data_to_tensor, index_dict, vstack_dicts, dict_element_len\n",
    "from ground_truth_maker import make_yolo_masks_for_dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class BBoxIterator(object):\n",
    "    def __init__(self,kwargs,mode=\"tr\"):\n",
    "        self.kwargs = kwargs\n",
    "        self.mode = mode\n",
    "        #frame = inspect.currentframe()\n",
    "        self.seed = 7\n",
    "        self.camfiles = get_camfiles(self.kwargs[\"data_dir\"], self.kwargs[mode+ \"_years\"])[:self.kwargs[\"num_\" + mode + \"_days\"]]\n",
    "        self.file_ind = 0\n",
    "        self.chunk_ind =0 \n",
    "        self.data_chunk=[]\n",
    "        self.num_events = len(self.camfiles) * 4\n",
    "        self.events_open = 4 * self.kwargs[\"max_files_open\"]\n",
    "        if kwargs[\"im_dim\"] == 3:\n",
    "            assert kwargs[\"max_files_open\"] >= kwargs['3d_time_steps_per_example'] /  (kwargs[\"time_steps_per_file\"] / \n",
    "                                            kwargs[\"time_step_sample_frequency\"]),\"increase max_files open!\"\n",
    "       \n",
    "    \n",
    "    def iterate_chunks(self,batch_size=128):\n",
    "        chunk_index = 0\n",
    "        events_read =0 \n",
    "        chunk = self.get_next_chunk()\n",
    "        while events_read < self.num_events:\n",
    "            if chunk_index + batch_size > dict_element_len(chunk):\n",
    "                chunk = self.finish_out_chunk_and_get_as_many_more_as_needed(chunk, \n",
    "                                                                                  chunk_index, \n",
    "                                                                                  batch_size)\n",
    "                #back to 0 b/c we have a brand new chunk\n",
    "                chunk_index = 0\n",
    "            \n",
    "            if events_read + batch_size > self.num_events:\n",
    "                sm_batch_size = self.num_events - events_read\n",
    "                excerpt = slice(chunk_index, chunk_index + sm_batch_size)\n",
    "            \n",
    "            else:\n",
    "\n",
    "                #otherwise just read an excerpt from the current chunk\n",
    "                excerpt = slice(chunk_index, chunk_index + batch_size)\n",
    "            \n",
    "            \n",
    "            chunk_index += batch_size\n",
    "            events_read += batch_size\n",
    "            \n",
    "            ch = index_dict(chunk, excerpt)\n",
    "            \n",
    "            yield ch\n",
    "            \n",
    "    \n",
    "    \n",
    "    def finish_out_chunk_and_get_as_many_more_as_needed(self, chunk, ix, batch_size):\n",
    "        chunk_len = dict_element_len(chunk)\n",
    "        tmp = index_dict(chunk, slice(ix, chunk_len ))\n",
    "        chunk = self.get_chunks_until_at_capacity(batch_size - dict_element_len(tmp))\n",
    "        chunk = vstack_dicts(tmp,chunk)\n",
    "        return chunk\n",
    "        \n",
    "    def get_chunks_until_at_capacity(self,batch_size):\n",
    "        tmp = self.get_next_chunk()\n",
    "        while  dict_element_len(tmp) < batch_size:\n",
    "            chunk = self.get_next_chunk()\n",
    "            tmp = vstack_dicts(tmp,chunk)\n",
    "        return tmp\n",
    "\n",
    "    def get_next_chunk(self):\n",
    "        mfo = kwargs[\"max_files_open\"]\n",
    "        \n",
    "        #if we are starting back up again shuffle everything\n",
    "        if self.file_ind < mfo:\n",
    "            if self.kwargs[\"shuffle\"] and not self.kwargs[\"3D\"]:\n",
    "                self.camfiles = self.camfiles.shuffle()\n",
    "        \n",
    "        #get next chunk of files\n",
    "        filenames = self.camfiles[self.file_ind: mfo + self.file_ind]\n",
    "        \n",
    "        #increment index to start with (modulo for circular effect)\n",
    "        self.file_ind = (self.file_ind + kwargs[\"max_files_open\"] ) % len(self.camfiles)\n",
    "        \n",
    "        return self._get_next_chunk(filenames)\n",
    "        \n",
    "        \n",
    "    def _get_next_chunk(self,filenames):\n",
    "        \n",
    "        data_chunk = self.grab_data_chunk(filenames)\n",
    "        \n",
    "        label_chunk = self.grab_label_chunk(filenames)\n",
    "        return {\"data\":data_chunk, \"label\": label_chunk}\n",
    "    \n",
    "    def grab_label_chunk(self,filenames):\n",
    "        \n",
    "        labels = make_yolo_masks_for_dataset(filenames[0],self.kwargs)\n",
    "        if len(filenames) > 1:\n",
    "            for f in filenames[1:]:\n",
    "                label = make_yolo_masks_for_dataset(f,self.kwargs)\n",
    "                labels = np.vstack((labels, label))\n",
    "                \n",
    "        if self.kwargs[\"im_dim\"] == 3:\n",
    "            time_steps = labels.shape[0]\n",
    "            time_steps_per_example = self.kwargs[\"3d_time_steps_per_example\"]\n",
    "            labels = labels.reshape(time_steps / time_steps_per_example, \n",
    "                                       time_steps_per_example, \n",
    "                                       labels.shape[-3], \n",
    "                                       labels.shape[-2], \n",
    "                                       labels.shape[-1] )\n",
    "            \n",
    "            \n",
    "        return labels\n",
    "        \n",
    "    \n",
    "    def grab_data_chunk(self, filenames):\n",
    "        \"\"\"grabs input data (converts filenames to numpy tensors)\n",
    "        returns len(filenames)*4, 16, 768,1152 array\"\"\"\n",
    "        \n",
    "        filenames = [join(self.kwargs[\"data_dir\"],f) for f in filenames]\n",
    "\n",
    "\n",
    "        dataset=MFDataset(filenames)\n",
    "        \n",
    "        tensor = convert_nc_data_to_tensor(dataset,self.kwargs)\n",
    " \n",
    "        return tensor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1, 16, 8, 768, 1152) (1, 8, 10, 12, 18)\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-8-f78cdec204b3>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      6\u001b[0m     \u001b[0mkwargs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'num_val_days'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m10\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      7\u001b[0m     \u001b[0mt\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 8\u001b[0;31m     \u001b[0;32mfor\u001b[0m \u001b[0mx\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mBBoxIterator\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mmode\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"val\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0miterate_chunks\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch_size\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      9\u001b[0m         \u001b[0;32mprint\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'data'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'label'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     10\u001b[0m     \u001b[0;32mprint\u001b[0m \u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0mt\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-6-feec0aa78500>\u001b[0m in \u001b[0;36miterate_chunks\u001b[0;34m(self, batch_size)\u001b[0m\n\u001b[1;32m     24\u001b[0m                 chunk = self.finish_out_chunk_and_get_as_many_more_as_needed(chunk, \n\u001b[1;32m     25\u001b[0m                                                                                   \u001b[0mchunk_index\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 26\u001b[0;31m                                                                                   batch_size)\n\u001b[0m\u001b[1;32m     27\u001b[0m                 \u001b[0;31m#back to 0 b/c we have a brand new chunk\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     28\u001b[0m                 \u001b[0mchunk_index\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-6-feec0aa78500>\u001b[0m in \u001b[0;36mfinish_out_chunk_and_get_as_many_more_as_needed\u001b[0;34m(self, chunk, ix, batch_size)\u001b[0m\n\u001b[1;32m     50\u001b[0m         \u001b[0mchunk_len\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdict_element_len\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mchunk\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     51\u001b[0m         \u001b[0mtmp\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mindex_dict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mchunk\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mslice\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mix\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mchunk_len\u001b[0m \u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 52\u001b[0;31m         \u001b[0mchunk\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_chunks_until_at_capacity\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch_size\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0mdict_element_len\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtmp\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     53\u001b[0m         \u001b[0mchunk\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mvstack_dicts\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtmp\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mchunk\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     54\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mchunk\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-6-feec0aa78500>\u001b[0m in \u001b[0;36mget_chunks_until_at_capacity\u001b[0;34m(self, batch_size)\u001b[0m\n\u001b[1;32m     55\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     56\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mget_chunks_until_at_capacity\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mbatch_size\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 57\u001b[0;31m         \u001b[0mtmp\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_next_chunk\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     58\u001b[0m         \u001b[0;32mwhile\u001b[0m  \u001b[0mdict_element_len\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtmp\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m<\u001b[0m \u001b[0mbatch_size\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     59\u001b[0m             \u001b[0mchunk\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_next_chunk\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-6-feec0aa78500>\u001b[0m in \u001b[0;36mget_next_chunk\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m     75\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfile_ind\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfile_ind\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"max_files_open\"\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m)\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcamfiles\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     76\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 77\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_get_next_chunk\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfilenames\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     78\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     79\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-6-feec0aa78500>\u001b[0m in \u001b[0;36m_get_next_chunk\u001b[0;34m(self, filenames)\u001b[0m\n\u001b[1;32m     80\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_get_next_chunk\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mfilenames\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     81\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 82\u001b[0;31m         \u001b[0mdata_chunk\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgrab_data_chunk\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfilenames\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     83\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     84\u001b[0m         \u001b[0mlabel_chunk\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgrab_label_chunk\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfilenames\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-6-feec0aa78500>\u001b[0m in \u001b[0;36mgrab_data_chunk\u001b[0;34m(self, filenames)\u001b[0m\n\u001b[1;32m    115\u001b[0m         \u001b[0mdataset\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mMFDataset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfilenames\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    116\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 117\u001b[0;31m         \u001b[0mtensor\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mconvert_nc_data_to_tensor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdataset\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    118\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    119\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mtensor\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/evan/hur-detect/notebooks/load_data/util.ipynb\u001b[0m in \u001b[0;36mconvert_nc_data_to_tensor\u001b[0;34m(dataset, kwargs)\u001b[0m\n",
      "\u001b[0;32mnetCDF4/_netCDF4.pyx\u001b[0m in \u001b[0;36mnetCDF4._netCDF4._Variable.__getitem__ (netCDF4/_netCDF4.c:76417)\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32mnetCDF4/_netCDF4.pyx\u001b[0m in \u001b[0;36mnetCDF4._netCDF4.Variable.__getitem__ (netCDF4/_netCDF4.c:41100)\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32mnetCDF4/_netCDF4.pyx\u001b[0m in \u001b[0;36mnetCDF4._netCDF4.Variable._toma (netCDF4/_netCDF4.c:42781)\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python2.7/dist-packages/numpy/core/_methods.pyc\u001b[0m in \u001b[0;36m_any\u001b[0;34m(a, axis, dtype, out, keepdims)\u001b[0m\n\u001b[1;32m     35\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mumr_prod\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ma\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maxis\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mout\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkeepdims\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     36\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 37\u001b[0;31m \u001b[0;32mdef\u001b[0m \u001b[0m_any\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ma\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maxis\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkeepdims\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     38\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mumr_any\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ma\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maxis\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mout\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkeepdims\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     39\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "if __name__ == \"__main__\":\n",
    "    sys.path.insert(0,\"/home/evan/hur-detect/scripts/\")\n",
    "    from configs import *\n",
    "    kwargs = process_kwargs()\n",
    "    kwargs[\"max_files_open\"] = 4\n",
    "    kwargs['num_val_days'] = 10\n",
    "    t = time.time()\n",
    "    for x in BBoxIterator(kwargs,mode=\"val\").iterate_chunks(batch_size=1):\n",
    "        print x['data'].shape, x['label'].shape\n",
    "    print time.time() - t"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# from matplotlib import pyplot as plt\n",
    "# %matplotlib inline\n",
    "\n",
    "# plt.imshow(x[\"data\"][0][1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
